import numpy as np
import tensorflow.compat.v2 as tf
tf.enable_v2_behavior()
import pandas as pd
from tensorflow import keras
from sklearn.preprocessing import StandardScaler
from sklearn.preprocessing import RobustScaler
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot
import plotly.graph_objects as go
import math
import seaborn as sns
from sklearn.metrics import mean_squared_error
np.random.seed(1)
tf.random.set_seed(1)
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, GRU, Dropout, RepeatVector, TimeDistributed
from keras import backend
MODELFILENAME = 'MODELS/LSTM_3h_TFM'
TIME_STEPS=18 #3h
CMODEL = LSTM
UNITS=40
DROPOUT=0.405
ACTIVATION='tanh'
OPTIMIZER='adam'
EPOCHS=68
BATCHSIZE=45
VALIDATIONSPLIT=0.1
# Code to read csv file into Colaboratory:
# from google.colab import files
# uploaded = files.upload()
# import io
# df = pd.read_csv(io.BytesIO(uploaded['SentDATA.csv']))
# Dataset is now stored in a Pandas Dataframe
df = pd.read_csv('../../data/dadesTFM.csv')
df.reset_index(inplace=True)
df['Time'] = pd.to_datetime(df['Time'])
df = df.set_index('Time')
columns = ['PM1','PM25','PM10','PM1ATM','PM25ATM','PM10ATM']
df1 = df.copy();
df1 = df1.rename(columns={"PM 1":"PM1","PM 2.5":"PM25","PM 10":"PM10","PM 1 ATM":"PM1ATM","PM 2.5 ATM":"PM25ATM","PM 10 ATM":"PM10ATM"})
df1['PM1'] = df['PM 1'].astype(np.float32)
df1['PM25'] = df['PM 2.5'].astype(np.float32)
df1['PM10'] = df['PM 10'].astype(np.float32)
df1['PM1ATM'] = df['PM 1 ATM'].astype(np.float32)
df1['PM25ATM'] = df['PM 2.5 ATM'].astype(np.float32)
df1['PM10ATM'] = df['PM 10 ATM'].astype(np.float32)
df2 = df1.copy()
train_size = int(len(df2) * 0.8)
test_size = len(df2) - train_size
train, test = df2.iloc[0:train_size], df2.iloc[train_size:len(df2)]
train.shape, test.shape
((3117, 7), (780, 7))
#Standardize the data
for col in columns:
scaler = StandardScaler()
train[col] = scaler.fit_transform(train[[col]])
<ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]]) <ipython-input-6-83cecdbc25f8>:4: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy train[col] = scaler.fit_transform(train[[col]])
def create_sequences(X, y, time_steps=TIME_STEPS):
Xs, ys = [], []
for i in range(len(X)-time_steps):
Xs.append(X.iloc[i:(i+time_steps)].values)
ys.append(y.iloc[i+time_steps])
return np.array(Xs), np.array(ys)
X_train, y_train = create_sequences(train[[columns[1]]], train[columns[1]])
#X_test, y_test = create_sequences(test[[columns[1]]], test[columns[1]])
print(f'X_train shape: {X_train.shape}')
print(f'y_train shape: {y_train.shape}')
X_train shape: (3099, 18, 1) y_train shape: (3099,)
#afegir nova mètrica
def rmse(y_true, y_pred):
return backend.sqrt(backend.mean(backend.square(y_pred - y_true), axis=-1))
model = Sequential()
model.add(CMODEL(units = UNITS, return_sequences=True, input_shape=(X_train.shape[1], X_train.shape[2])))
model.add(Dropout(rate=DROPOUT))
model.add(TimeDistributed(Dense(1,kernel_initializer='normal',activation=ACTIVATION)))
model.compile(optimizer=OPTIMIZER, loss='mae',metrics=['mse',rmse])
model.summary()
Model: "sequential" _________________________________________________________________ Layer (type) Output Shape Param # ================================================================= lstm (LSTM) (None, 18, 40) 6720 _________________________________________________________________ dropout (Dropout) (None, 18, 40) 0 _________________________________________________________________ time_distributed (TimeDistri (None, 18, 1) 41 ================================================================= Total params: 6,761 Trainable params: 6,761 Non-trainable params: 0 _________________________________________________________________
history = model.fit(X_train, y_train, epochs=EPOCHS, batch_size=BATCHSIZE, validation_split=VALIDATIONSPLIT,
callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=5, mode='min')], shuffle=False)
Epoch 1/68 62/62 [==============================] - 1s 13ms/step - loss: 0.7079 - mse: 0.8665 - rmse: 0.7141 - val_loss: 0.5262 - val_mse: 0.3239 - val_rmse: 0.5553 Epoch 2/68 62/62 [==============================] - 0s 6ms/step - loss: 0.5256 - mse: 0.5575 - rmse: 0.5659 - val_loss: 0.3240 - val_mse: 0.1613 - val_rmse: 0.3904 Epoch 3/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4836 - mse: 0.5033 - rmse: 0.5262 - val_loss: 0.2626 - val_mse: 0.1190 - val_rmse: 0.3322 Epoch 4/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4617 - mse: 0.4733 - rmse: 0.5073 - val_loss: 0.2378 - val_mse: 0.1011 - val_rmse: 0.3039 Epoch 5/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4489 - mse: 0.4591 - rmse: 0.4951 - val_loss: 0.2183 - val_mse: 0.0885 - val_rmse: 0.2820 Epoch 6/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4401 - mse: 0.4492 - rmse: 0.4860 - val_loss: 0.2074 - val_mse: 0.0806 - val_rmse: 0.2669 Epoch 7/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4340 - mse: 0.4425 - rmse: 0.4795 - val_loss: 0.1962 - val_mse: 0.0738 - val_rmse: 0.2532 Epoch 8/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4299 - mse: 0.4377 - rmse: 0.4749 - val_loss: 0.1909 - val_mse: 0.0697 - val_rmse: 0.2441 Epoch 9/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4260 - mse: 0.4339 - rmse: 0.4707 - val_loss: 0.1840 - val_mse: 0.0657 - val_rmse: 0.2349 Epoch 10/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4231 - mse: 0.4309 - rmse: 0.4673 - val_loss: 0.1821 - val_mse: 0.0633 - val_rmse: 0.2291 Epoch 11/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4206 - mse: 0.4282 - rmse: 0.4645 - val_loss: 0.1766 - val_mse: 0.0604 - val_rmse: 0.2217 Epoch 12/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4185 - mse: 0.4261 - rmse: 0.4619 - val_loss: 0.1725 - val_mse: 0.0581 - val_rmse: 0.2157 Epoch 13/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4170 - mse: 0.4246 - rmse: 0.4599 - val_loss: 0.1704 - val_mse: 0.0565 - val_rmse: 0.2113 Epoch 14/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4160 - mse: 0.4238 - rmse: 0.4585 - val_loss: 0.1681 - val_mse: 0.0550 - val_rmse: 0.2071 Epoch 15/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4147 - mse: 0.4227 - rmse: 0.4569 - val_loss: 0.1656 - val_mse: 0.0535 - val_rmse: 0.2029 Epoch 16/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4140 - mse: 0.4214 - rmse: 0.4556 - val_loss: 0.1640 - val_mse: 0.0524 - val_rmse: 0.1995 Epoch 17/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4127 - mse: 0.4200 - rmse: 0.4541 - val_loss: 0.1620 - val_mse: 0.0513 - val_rmse: 0.1961 Epoch 18/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4120 - mse: 0.4199 - rmse: 0.4535 - val_loss: 0.1595 - val_mse: 0.0502 - val_rmse: 0.1926 Epoch 19/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4110 - mse: 0.4185 - rmse: 0.4521 - val_loss: 0.1594 - val_mse: 0.0498 - val_rmse: 0.1911 Epoch 20/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4105 - mse: 0.4181 - rmse: 0.4513 - val_loss: 0.1590 - val_mse: 0.0492 - val_rmse: 0.1891 Epoch 21/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4099 - mse: 0.4174 - rmse: 0.4508 - val_loss: 0.1568 - val_mse: 0.0482 - val_rmse: 0.1860 Epoch 22/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4090 - mse: 0.4165 - rmse: 0.4495 - val_loss: 0.1571 - val_mse: 0.0479 - val_rmse: 0.1849 Epoch 23/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4088 - mse: 0.4167 - rmse: 0.4495 - val_loss: 0.1540 - val_mse: 0.0470 - val_rmse: 0.1818 Epoch 24/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4091 - mse: 0.4167 - rmse: 0.4497 - val_loss: 0.1557 - val_mse: 0.0471 - val_rmse: 0.1818 Epoch 25/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4074 - mse: 0.4149 - rmse: 0.4478 - val_loss: 0.1521 - val_mse: 0.0461 - val_rmse: 0.1783 Epoch 26/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4084 - mse: 0.4155 - rmse: 0.4485 - val_loss: 0.1548 - val_mse: 0.0464 - val_rmse: 0.1792 Epoch 27/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4075 - mse: 0.4152 - rmse: 0.4479 - val_loss: 0.1507 - val_mse: 0.0452 - val_rmse: 0.1752 Epoch 28/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4071 - mse: 0.4147 - rmse: 0.4474 - val_loss: 0.1506 - val_mse: 0.0450 - val_rmse: 0.1743 Epoch 29/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4068 - mse: 0.4148 - rmse: 0.4472 - val_loss: 0.1506 - val_mse: 0.0448 - val_rmse: 0.1735 Epoch 30/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4064 - mse: 0.4139 - rmse: 0.4467 - val_loss: 0.1496 - val_mse: 0.0445 - val_rmse: 0.1724 Epoch 31/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4064 - mse: 0.4142 - rmse: 0.4465 - val_loss: 0.1501 - val_mse: 0.0445 - val_rmse: 0.1722 Epoch 32/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4065 - mse: 0.4139 - rmse: 0.4465 - val_loss: 0.1488 - val_mse: 0.0441 - val_rmse: 0.1707 Epoch 33/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4058 - mse: 0.4132 - rmse: 0.4458 - val_loss: 0.1492 - val_mse: 0.0441 - val_rmse: 0.1704 Epoch 34/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4054 - mse: 0.4129 - rmse: 0.4451 - val_loss: 0.1461 - val_mse: 0.0433 - val_rmse: 0.1676 Epoch 35/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4052 - mse: 0.4127 - rmse: 0.4451 - val_loss: 0.1476 - val_mse: 0.0436 - val_rmse: 0.1682 Epoch 36/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4050 - mse: 0.4127 - rmse: 0.4448 - val_loss: 0.1464 - val_mse: 0.0433 - val_rmse: 0.1669 Epoch 37/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4047 - mse: 0.4124 - rmse: 0.4446 - val_loss: 0.1470 - val_mse: 0.0431 - val_rmse: 0.1667 Epoch 38/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4043 - mse: 0.4119 - rmse: 0.4440 - val_loss: 0.1466 - val_mse: 0.0431 - val_rmse: 0.1660 Epoch 39/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4042 - mse: 0.4116 - rmse: 0.4440 - val_loss: 0.1452 - val_mse: 0.0427 - val_rmse: 0.1646 Epoch 40/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4042 - mse: 0.4118 - rmse: 0.4441 - val_loss: 0.1455 - val_mse: 0.0428 - val_rmse: 0.1647 Epoch 41/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4040 - mse: 0.4116 - rmse: 0.4439 - val_loss: 0.1453 - val_mse: 0.0427 - val_rmse: 0.1642 Epoch 42/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4037 - mse: 0.4109 - rmse: 0.4436 - val_loss: 0.1449 - val_mse: 0.0425 - val_rmse: 0.1633 Epoch 43/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4034 - mse: 0.4112 - rmse: 0.4432 - val_loss: 0.1437 - val_mse: 0.0421 - val_rmse: 0.1622 Epoch 44/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4036 - mse: 0.4114 - rmse: 0.4434 - val_loss: 0.1457 - val_mse: 0.0426 - val_rmse: 0.1637 Epoch 45/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4032 - mse: 0.4109 - rmse: 0.4430 - val_loss: 0.1427 - val_mse: 0.0420 - val_rmse: 0.1613 Epoch 46/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4036 - mse: 0.4110 - rmse: 0.4434 - val_loss: 0.1465 - val_mse: 0.0428 - val_rmse: 0.1641 Epoch 47/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4029 - mse: 0.4104 - rmse: 0.4430 - val_loss: 0.1426 - val_mse: 0.0419 - val_rmse: 0.1608 Epoch 48/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4028 - mse: 0.4105 - rmse: 0.4429 - val_loss: 0.1425 - val_mse: 0.0419 - val_rmse: 0.1606 Epoch 49/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4029 - mse: 0.4101 - rmse: 0.4431 - val_loss: 0.1440 - val_mse: 0.0422 - val_rmse: 0.1617 Epoch 50/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4028 - mse: 0.4101 - rmse: 0.4428 - val_loss: 0.1435 - val_mse: 0.0421 - val_rmse: 0.1612 Epoch 51/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4026 - mse: 0.4101 - rmse: 0.4429 - val_loss: 0.1424 - val_mse: 0.0418 - val_rmse: 0.1602 Epoch 52/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4024 - mse: 0.4101 - rmse: 0.4426 - val_loss: 0.1433 - val_mse: 0.0420 - val_rmse: 0.1609 Epoch 53/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4025 - mse: 0.4101 - rmse: 0.4426 - val_loss: 0.1419 - val_mse: 0.0417 - val_rmse: 0.1595 Epoch 54/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4022 - mse: 0.4097 - rmse: 0.4423 - val_loss: 0.1433 - val_mse: 0.0419 - val_rmse: 0.1603 Epoch 55/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4022 - mse: 0.4097 - rmse: 0.4423 - val_loss: 0.1420 - val_mse: 0.0417 - val_rmse: 0.1594 Epoch 56/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4018 - mse: 0.4094 - rmse: 0.4422 - val_loss: 0.1416 - val_mse: 0.0417 - val_rmse: 0.1592 Epoch 57/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4022 - mse: 0.4096 - rmse: 0.4424 - val_loss: 0.1418 - val_mse: 0.0418 - val_rmse: 0.1594 Epoch 58/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4022 - mse: 0.4098 - rmse: 0.4425 - val_loss: 0.1426 - val_mse: 0.0418 - val_rmse: 0.1598 Epoch 59/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4017 - mse: 0.4092 - rmse: 0.4420 - val_loss: 0.1421 - val_mse: 0.0417 - val_rmse: 0.1593 Epoch 60/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4022 - mse: 0.4093 - rmse: 0.4425 - val_loss: 0.1438 - val_mse: 0.0421 - val_rmse: 0.1607 Epoch 61/68 62/62 [==============================] - 0s 7ms/step - loss: 0.4016 - mse: 0.4091 - rmse: 0.4420 - val_loss: 0.1415 - val_mse: 0.0416 - val_rmse: 0.1587 Epoch 62/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4020 - mse: 0.4093 - rmse: 0.4424 - val_loss: 0.1413 - val_mse: 0.0416 - val_rmse: 0.1586 Epoch 63/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4018 - mse: 0.4094 - rmse: 0.4424 - val_loss: 0.1411 - val_mse: 0.0416 - val_rmse: 0.1585 Epoch 64/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4016 - mse: 0.4088 - rmse: 0.4418 - val_loss: 0.1414 - val_mse: 0.0416 - val_rmse: 0.1587 Epoch 65/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4016 - mse: 0.4086 - rmse: 0.4418 - val_loss: 0.1405 - val_mse: 0.0414 - val_rmse: 0.1578 Epoch 66/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4015 - mse: 0.4091 - rmse: 0.4417 - val_loss: 0.1403 - val_mse: 0.0414 - val_rmse: 0.1577 Epoch 67/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4011 - mse: 0.4084 - rmse: 0.4415 - val_loss: 0.1403 - val_mse: 0.0413 - val_rmse: 0.1575 Epoch 68/68 62/62 [==============================] - 0s 6ms/step - loss: 0.4010 - mse: 0.4086 - rmse: 0.4413 - val_loss: 0.1404 - val_mse: 0.0413 - val_rmse: 0.1576
import matplotlib.pyplot as plt
plt.plot(history.history['loss'], label='MAE Training loss')
plt.plot(history.history['val_loss'], label='MAE Validation loss')
plt.plot(history.history['mse'], label='MSE Training loss')
plt.plot(history.history['val_mse'], label='MSE Validation loss')
plt.plot(history.history['rmse'], label='RMSE Training loss')
plt.plot(history.history['val_rmse'], label='RMSE Validation loss')
plt.legend();
X_train_pred = model.predict(X_train, verbose=0)
train_mae_loss = np.mean(np.abs(X_train_pred - X_train), axis=1)
plt.hist(train_mae_loss, bins=50)
plt.xlabel('Train MAE loss')
plt.ylabel('Number of Samples');
def evaluate_prediction(predictions, actual, model_name):
errors = predictions - actual
mse = np.square(errors).mean()
rmse = np.sqrt(mse)
mae = np.abs(errors).mean()
print(model_name + ':')
print('Mean Absolute Error: {:.4f}'.format(mae))
print('Root Mean Square Error: {:.4f}'.format(rmse))
print('Mean Square Error: {:.4f}'.format(mse))
print('')
return mae,rmse,mse
mae,rmse,mse = evaluate_prediction(X_train_pred, X_train,"LSTM")
LSTM: Mean Absolute Error: 0.1935 Root Mean Square Error: 0.4377 Mean Square Error: 0.1916
model.save(MODELFILENAME+'.h5')
#càlcul del threshold de test
def calculate_threshold(X_test, X_test_pred):
distance = np.sqrt(np.mean(np.square(X_test_pred - X_test),axis=1))
"""Sorting the scores/diffs and using a 0.80 as cutoff value to pick the threshold"""
distance.sort();
cut_off = int(0.95 * len(distance));
threshold = distance[cut_off];
return threshold
for col in columns:
print ("####################### "+col +" ###########################")
#Standardize the test data
scaler = StandardScaler()
test_cpy = test.copy()
test[col] = scaler.fit_transform(test[[col]])
#creem seqüencia amb finestra temporal per les dades de test
X_test1, y_test1 = create_sequences(test[[col]], test[col])
print(f'Testing shape: {X_test1.shape}')
#evaluem el model
eval = model.evaluate(X_test1, y_test1)
print("evaluate: ",eval)
#predim el model
X_test1_pred = model.predict(X_test1, verbose=0)
evaluate_prediction(X_test1_pred, X_test1,"LSTM")
#càlcul del mae_loss
test1_mae_loss = np.mean(np.abs(X_test1_pred - X_test1), axis=1)
test1_rmse_loss = np.sqrt(np.mean(np.square(X_test1_pred - X_test1),axis=1))
# reshaping test prediction
X_test1_predReshape = X_test1_pred.reshape((X_test1_pred.shape[0] * X_test1_pred.shape[1]), X_test1_pred.shape[2])
# reshaping test data
X_test1Reshape = X_test1.reshape((X_test1.shape[0] * X_test1.shape[1]), X_test1.shape[2])
threshold_test = calculate_threshold(X_test1Reshape,X_test1_predReshape)
test1_score_df = pd.DataFrame(test[TIME_STEPS:])
test1_score_df['loss'] = test1_rmse_loss.reshape((-1))
test1_score_df['threshold'] = threshold_test
test1_score_df['anomaly'] = test1_score_df['loss'] > test1_score_df['threshold']
test1_score_df[col] = test[TIME_STEPS:][col]
#gràfic test lost i threshold
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['loss'], name='Test loss'))
fig.add_trace(go.Scatter(x=test1_score_df.index, y=test1_score_df['threshold'], name='Threshold'))
fig.update_layout(showlegend=True, title='Test loss vs. Threshold')
fig.show()
#Posem les anomalies en un array
anomalies1 = test1_score_df.loc[test1_score_df['anomaly'] == True]
anomalies1.shape
print('anomalies: ',anomalies1.shape); print();
#Gràfic dels punts i de les anomalíes amb els valors de dades transformades per verificar que la normalització que s'ha fet no distorssiona les dades
fig = go.Figure()
fig.add_trace(go.Scatter(x=test1_score_df.index, y=scaler.inverse_transform(test1_score_df[col]), name=col))
fig.add_trace(go.Scatter(x=anomalies1.index, y=scaler.inverse_transform(anomalies1[col]), mode='markers', name='Anomaly'))
fig.update_layout(showlegend=True, title='Detected anomalies')
fig.show()
print ("######################################################")
####################### PM1 ########################### Testing shape: (762, 18, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy test[col] = scaler.fit_transform(test[[col]])
24/24 [==============================] - 0s 2ms/step - loss: 0.4763 - mse: 0.7983 - rmse: 0.5398 evaluate: [0.47625428438186646, 0.798281192779541, 0.5397927165031433] LSTM: Mean Absolute Error: 0.1856 Root Mean Square Error: 0.5906 Mean Square Error: 0.3488
anomalies: (59, 10)
###################################################### ####################### PM25 ########################### Testing shape: (762, 18, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 2ms/step - loss: 0.4939 - mse: 0.7457 - rmse: 0.5569 evaluate: [0.49387282133102417, 0.7457113862037659, 0.5568555593490601] LSTM: Mean Absolute Error: 0.1981 Root Mean Square Error: 0.5497 Mean Square Error: 0.3021
anomalies: (94, 10)
###################################################### ####################### PM10 ########################### Testing shape: (762, 18, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 2ms/step - loss: 0.5028 - mse: 0.7218 - rmse: 0.5679 evaluate: [0.5027919411659241, 0.7217568755149841, 0.5678677558898926] LSTM: Mean Absolute Error: 0.2045 Root Mean Square Error: 0.5120 Mean Square Error: 0.2621
anomalies: (54, 10)
###################################################### ####################### PM1ATM ########################### Testing shape: (762, 18, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 2ms/step - loss: 0.5111 - mse: 0.7536 - rmse: 0.5793 evaluate: [0.5111464262008667, 0.7536243200302124, 0.5793437957763672] LSTM: Mean Absolute Error: 0.1996 Root Mean Square Error: 0.5057 Mean Square Error: 0.2557
anomalies: (61, 10)
###################################################### ####################### PM25ATM ########################### Testing shape: (762, 18, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 3ms/step - loss: 0.5059 - mse: 0.7587 - rmse: 0.5734 evaluate: [0.5058514475822449, 0.758709192276001, 0.5734078884124756] LSTM: Mean Absolute Error: 0.1975 Root Mean Square Error: 0.5177 Mean Square Error: 0.2680
anomalies: (61, 10)
###################################################### ####################### PM10ATM ########################### Testing shape: (762, 18, 1)
<ipython-input-17-48420fb1aa44>:8: SettingWithCopyWarning: A value is trying to be set on a copy of a slice from a DataFrame. Try using .loc[row_indexer,col_indexer] = value instead See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
24/24 [==============================] - 0s 2ms/step - loss: 0.5002 - mse: 0.7277 - rmse: 0.5645 evaluate: [0.5001875162124634, 0.7277060747146606, 0.5645371079444885] LSTM: Mean Absolute Error: 0.2025 Root Mean Square Error: 0.5236 Mean Square Error: 0.2742
anomalies: (61, 10)
######################################################